{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.training import numpy_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x, axis=1):\n",
    "    return np.exp(x) / np.sum(np.exp(x),axis=axis, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ece(outputs, labels, name=\"\", bins=np.arange(0.1, 1.05, 0.05), print_name=False, background_label = 11):\n",
    "\n",
    "    if outputs is None:\n",
    "        ece = None\n",
    "    else:\n",
    "        confidences = np.max(outputs, 1)\n",
    "        predictions = np.argmax(outputs,1)\n",
    "        bin_lowers = bins[:-1]\n",
    "        bin_uppers = bins[1:]\n",
    "        \n",
    "        accuracies = predictions == labels \n",
    "\n",
    "        #ece = Variable(torch.zeros(1)).type_as(confidences)\n",
    "        ece = 0.0\n",
    "        for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):\n",
    "            # Calculated |confidence - accuracy| in each bin\n",
    "            in_bin = (confidences > bin_lower) * (confidences < bin_upper)\n",
    "            prop_in_bin = in_bin.mean()\n",
    "            if prop_in_bin > 0:\n",
    "                accuracy_in_bin = accuracies[in_bin].mean()\n",
    "                avg_confidence_in_bin = confidences[in_bin].mean()\n",
    "                ece += np.abs(avg_confidence_in_bin-accuracy_in_bin) * prop_in_bin\n",
    "    if print_name:\n",
    "        print(name, end=\": \")\n",
    "    if ece is not None:\n",
    "        print(\"%.3f\"%ece)\n",
    "    else:\n",
    "        print(\"-\")\n",
    "    return ece\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_mIOU_acc(loc, output_probs = True, background_class = 11):\n",
    "    npz_arr = np.load(loc)\n",
    "    \n",
    "    if output_probs:\n",
    "        pred_probs = npz_arr['predictions']\n",
    "    else:\n",
    "        pred_probs = softmax(npz_arr['preds'],1)\n",
    "    targets = npz_arr['targets']\n",
    "    \n",
    "    I, U, acc = numpy_metrics(pred_probs, targets)\n",
    "    mIOU = np.mean(I / U)\n",
    "    \n",
    "    filtered_targets = targets.reshape(-1)\n",
    "    background_labels = filtered_targets == background_class\n",
    "    filtered_targets = filtered_targets[~background_labels]\n",
    "    \n",
    "    filtered_pred_probs = pred_probs.reshape(-1, pred_probs.shape[1])\n",
    "    print(filtered_pred_probs.shape, filtered_pred_probs.sum(1).mean())\n",
    "    filtered_pred_probs = filtered_pred_probs[~background_labels, :]\n",
    "    \n",
    "    ece_val = ece(filtered_pred_probs, filtered_targets, print_name=False)\n",
    "    print(acc, mIOU, ece_val, loc)\n",
    "    return acc, mIOU, ece_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40262400, 11) 1.000000007170505\n",
      "0.089\n",
      "0.900859621513635 0.6231925359426722 0.08924742584234689 /home/wesley/Desktop/nfs01_tesla/swa_uncertainties/segmentation/test_preds/dropout_new/preds.npz\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.900859621513635, 0.6231925359426722, 0.08924742584234689)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_mIOU_acc('/home/wesley/Desktop/nfs01_tesla/swa_uncertainties/segmentation/test_preds/dropout_new/preds.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40262400, 11) 1.000000014855937\n",
      "0.091\n",
      "0.9101354251839606 0.63325866789444 0.09144399240896578 /home/wesley/Desktop/nfs01_tesla/swa_uncertainties/segmentation/test_preds/swag_new/preds.npz\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9101354251839606, 0.63325866789444, 0.09144399240896578)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_mIOU_acc('/home/wesley/Desktop/nfs01_tesla/swa_uncertainties/segmentation/test_preds/swag_new/preds.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40262400, 11) 0.9999993\n",
      "0.092\n",
      "0.9088837405143834 0.6325884654568185 0.09178933595602778 /home/wesley/Desktop/tesla/swa_uncertainties/experiments/segmentation/swag_980.npz\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9088837405143834, 0.6325884654568185, 0.09178933595602778)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_mIOU_acc('/home/wesley/Desktop/tesla/swa_uncertainties/experiments/segmentation/swag_980.npz', \n",
    "                 output_probs = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40262400, 11) 0.99999833\n",
      "0.091\n",
      "0.9105683390670791 0.6459073234037088 0.0909109173528229 /home/wesley/Desktop/tesla/swa_uncertainties/experiments/segmentation/sgd_850_final.npz\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9105683390670791, 0.6459073234037088, 0.0909109173528229)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_mIOU_acc('/home/wesley/Desktop/tesla/swa_uncertainties/experiments/segmentation/sgd_850_final.npz', \n",
    "                 output_probs = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
